{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8b504ed7",
   "metadata": {},
   "source": [
    "## Description:\n",
    "搭建MIIND模型，完成新闻推荐数据集下的召回实验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3f5d0745",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import pickle\n",
    "import random\n",
    "from datetime import datetime\n",
    "from collections import defaultdict\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# 从utils里面导入函数\n",
    "from utils import gen_data_set, gen_model_input, train_mind_model\n",
    "from utils import get_embeddings, get_mind_recall_res\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0833b7f",
   "metadata": {},
   "source": [
    "## 导入数据，划分呢数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a2744b07",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../data_process'\n",
    "data = pd.read_csv(os.path.join(data_path, 'train_data.csv'), index_col=0, parse_dates=['expo_time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "61be5f24",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择出需要用到的列\n",
    "use_cols = ['user_id', 'article_id', 'expo_time', 'net_status', 'exop_position', 'duration', 'device', 'city', 'age', 'gender', 'click']\n",
    "data_new = data[use_cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2739da22",
   "metadata": {},
   "source": [
    "## 划分测试集和训练集 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fa750804",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 按照用户分组，然后把最后一个item拿出来\n",
    "click_df = data_new[data_new['click']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "443ba6c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_hist_and_last_click(all_click):\n",
    "    all_click = all_click.sort_values(by=['user_id', 'expo_time'])\n",
    "    click_last_df = all_click.groupby('user_id').tail(1)\n",
    "    \n",
    "    # 如果用户只有一个点击，hist为空了，会导致训练的时候这个用户不可见，此时默认泄露一下\n",
    "    def hist_func(user_df):\n",
    "        if len(user_df) == 1:\n",
    "            return user_df\n",
    "        else:\n",
    "            return user_df[:-1]\n",
    "\n",
    "    click_hist_df = all_click.groupby('user_id').apply(hist_func).reset_index(drop=True)\n",
    "\n",
    "    return click_hist_df, click_last_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b3e52d53",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_click_hist_df, user_click_last_df = get_hist_and_last_click(click_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6a51827c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>article_id</th>\n",
       "      <th>expo_time</th>\n",
       "      <th>net_status</th>\n",
       "      <th>exop_position</th>\n",
       "      <th>duration</th>\n",
       "      <th>device</th>\n",
       "      <th>city</th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>click</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17340</td>\n",
       "      <td>464481478</td>\n",
       "      <td>2021-06-30 20:34:47</td>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "      <td>27</td>\n",
       "      <td>iPhoneX</td>\n",
       "      <td>上海</td>\n",
       "      <td>A_30_39</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17340</td>\n",
       "      <td>465148736</td>\n",
       "      <td>2021-07-02 19:35:03</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>49</td>\n",
       "      <td>iPhoneX</td>\n",
       "      <td>上海</td>\n",
       "      <td>A_30_39</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17340</td>\n",
       "      <td>464707540</td>\n",
       "      <td>2021-07-02 19:47:06</td>\n",
       "      <td>5</td>\n",
       "      <td>25</td>\n",
       "      <td>174</td>\n",
       "      <td>iPhoneX</td>\n",
       "      <td>上海</td>\n",
       "      <td>A_30_39</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17340</td>\n",
       "      <td>464993414</td>\n",
       "      <td>2021-07-02 19:47:06</td>\n",
       "      <td>5</td>\n",
       "      <td>27</td>\n",
       "      <td>11</td>\n",
       "      <td>iPhoneX</td>\n",
       "      <td>上海</td>\n",
       "      <td>A_30_39</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17340</td>\n",
       "      <td>465115022</td>\n",
       "      <td>2021-07-02 20:01:34</td>\n",
       "      <td>5</td>\n",
       "      <td>41</td>\n",
       "      <td>14</td>\n",
       "      <td>iPhoneX</td>\n",
       "      <td>上海</td>\n",
       "      <td>A_30_39</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  article_id           expo_time  net_status  exop_position  \\\n",
       "0    17340   464481478 2021-06-30 20:34:47           2             21   \n",
       "1    17340   465148736 2021-07-02 19:35:03           5             23   \n",
       "2    17340   464707540 2021-07-02 19:47:06           5             25   \n",
       "3    17340   464993414 2021-07-02 19:47:06           5             27   \n",
       "4    17340   465115022 2021-07-02 20:01:34           5             41   \n",
       "\n",
       "   duration   device city      age gender  click  \n",
       "0        27  iPhoneX   上海  A_30_39   male      1  \n",
       "1        49  iPhoneX   上海  A_30_39   male      1  \n",
       "2       174  iPhoneX   上海  A_30_39   male      1  \n",
       "3        11  iPhoneX   上海  A_30_39   male      1  \n",
       "4        14  iPhoneX   上海  A_30_39   male      1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_click_hist_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbd63c7b",
   "metadata": {},
   "source": [
    "## MIND召回"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1c4976cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mind_recall(data, topk=100, embedding_dim=8, his_seq_maxlen=50, negsample=0,\n",
    "                      batch_size=64, epochs=1, verbose=1, validation_split=0.0):\n",
    "    \"\"\"通过MIND模型，计算用户向量和文章向量\n",
    "    param: data: 用户日志数据\n",
    "    topk: 对于每个用户，召回多少篇文章\n",
    "    \"\"\"\n",
    "    user_id_raw = data[['user_id']].drop_duplicates('user_id')\n",
    "    doc_id_raw = data[['article_id']].drop_duplicates('article_id')\n",
    "    \n",
    "    # 类别数据编码   \n",
    "    base_features = ['user_id', 'article_id', 'city', 'age', 'gender']\n",
    "    feature_max_idx = {}\n",
    "    for f in base_features:\n",
    "        lbe = LabelEncoder()\n",
    "        data[f] = lbe.fit_transform(data[f])\n",
    "        feature_max_idx[f] = data[f].max() + 1\n",
    "        \n",
    "    # 构建用户id词典和doc的id词典，方便从用户idx找到原始的id\n",
    "    user_id_enc = data[['user_id']].drop_duplicates('user_id')\n",
    "    doc_id_enc = data[['article_id']].drop_duplicates('article_id')\n",
    "    user_idx_2_rawid = dict(zip(user_id_enc['user_id'], user_id_raw['user_id']))\n",
    "    doc_idx_2_rawid = dict(zip(doc_id_enc['article_id'], doc_id_raw['article_id']))\n",
    "    \n",
    "    # 保存下每篇文章的被点击数量， 方便后面高热文章的打压\n",
    "    doc_clicked_count_df = data.groupby('article_id')['click'].apply(lambda x: x.count()).reset_index()\n",
    "    doc_clicked_count_dict = dict(zip(doc_clicked_count_df['article_id'], doc_clicked_count_df['click']))\n",
    "    \n",
    "    if os.path.exists('data/train_model_input.pkl'):\n",
    "        train_model_input = pickle.load(open('data/train_model_input.pkl', 'rb'))\n",
    "        train_label = np.load('data/train_label.npy')\n",
    "        test_model_input = pickle.load(open('data/test_model_input.pkl', 'rb'))\n",
    "        test_label = np.load('data/test_label.npy')\n",
    "    else: \n",
    "        train_set, test_set = gen_data_set(data, doc_clicked_count_dict, negsample, control_users=False)\n",
    "\n",
    "        # 构造MIND模型的输入\n",
    "        train_model_input, train_label = gen_model_input(train_set, his_seq_maxlen)\n",
    "        test_model_input, test_label = gen_model_input(test_set, his_seq_maxlen)\n",
    "    \n",
    "        # 保存一份输入直接，要不然每次都得这么构造输入太慢了\n",
    "        pickle.dump(train_model_input, open('data/train_model_input.pkl', 'wb'))\n",
    "        pickle.dump(test_model_input, open('data/test_model_input.pkl', 'wb'))\n",
    "        np.save('data/train_label.npy', train_label)\n",
    "        np.save('data/test_label.npy', test_label)\n",
    "    \n",
    "    # 构建模型并完成训练\n",
    "    model = train_mind_model(train_model_input, train_label, embedding_dim, feature_max_idx, his_seq_maxlen, batch_size, epochs, verbose, validation_split)\n",
    "    \n",
    "    # 获得用户embedding和doc的embedding， 并进行保存\n",
    "    user_embs, doc_embs = get_embeddings(model, test_model_input, user_idx_2_rawid, doc_idx_2_rawid)\n",
    "    \n",
    "    # MIND模型这里有k个兴趣向量，所以要分开进行召回\n",
    "    user_embs1 = user_embs[:, 0, :]\n",
    "    user_embs2 = user_embs[:, 1, :]\n",
    "    \n",
    "    # 对每个用户，拿到召回结果并返回回来\n",
    "    user_recall_items_dict1 = get_mind_recall_res(user_embs1, doc_embs, user_idx_2_rawid, doc_idx_2_rawid, topk)\n",
    "    user_recall_items_dict2 = get_mind_recall_res(user_embs1, doc_embs, user_idx_2_rawid, doc_idx_2_rawid, topk)\n",
    "    \n",
    "    # 合并，当然我这里没有去重\n",
    "#     user_recall_items_dict = defaultdict(list)\n",
    "#     for user in user_recall_items_dict1:\n",
    "#         user_recall_items_dict[user] = user_recall_items_dict1[user] + user_recall_items_dict2[user]\n",
    "    \n",
    "    return user_recall_items_dict1, user_recall_items_dict2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "15639da1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 476201 samples\n",
      "476201/476201 [==============================] - 1087s 2ms/sample - loss: 0.3508\n"
     ]
    }
   ],
   "source": [
    "user_recall_doc_dict1, user_recall_doc_dict2 = mind_recall(user_click_hist_df, negsample=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "09a18691",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 20000)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(user_recall_doc_dict1), len(user_recall_doc_dict2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48cf6061",
   "metadata": {},
   "source": [
    "## 定义MIND模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c2558c29",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 依次评估召回的前10, 20, 30, 40, 50个文章中的击中率\n",
    "def metrics_recall(user_recall_items_dict, trn_last_click_df, topk=100):\n",
    "    last_click_item_dict = dict(zip(trn_last_click_df['user_id'], trn_last_click_df['article_id']))\n",
    "    user_num = len(user_recall_items_dict)\n",
    "    \n",
    "    for k in range(50, topk+1, 50):\n",
    "        hit_num = 0\n",
    "        for user, item_list in user_recall_items_dict.items():\n",
    "            if user in last_click_item_dict:\n",
    "                # 获取前k个召回的结果\n",
    "                tmp_recall_items = [x[0] for x in user_recall_items_dict[user][:k]]\n",
    "                if last_click_item_dict[user] in set(tmp_recall_items):\n",
    "                    hit_num += 1\n",
    "        \n",
    "        hit_rate = round(hit_num * 1.0 / user_num, 5)\n",
    "        print(' topk: ', k, ' : ', 'hit_num: ', hit_num, 'hit_rate: ', hit_rate, 'user_num : ', user_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cfb70a5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " topk:  50  :  hit_num:  28 hit_rate:  0.0014 user_num :  20000\n",
      " topk:  100  :  hit_num:  67 hit_rate:  0.00335 user_num :  20000\n",
      " topk:  150  :  hit_num:  67 hit_rate:  0.00335 user_num :  20000\n",
      " topk:  200  :  hit_num:  67 hit_rate:  0.00335 user_num :  20000\n"
     ]
    }
   ],
   "source": [
    "metrics_recall(user_recall_doc_dict1, user_click_last_df, topk=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "72064856",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " topk:  50  :  hit_num:  21 hit_rate:  0.00105 user_num :  20000\n",
      " topk:  100  :  hit_num:  64 hit_rate:  0.0032 user_num :  20000\n",
      " topk:  150  :  hit_num:  64 hit_rate:  0.0032 user_num :  20000\n",
      " topk:  200  :  hit_num:  64 hit_rate:  0.0032 user_num :  20000\n"
     ]
    }
   ],
   "source": [
    "metrics_recall(user_recall_doc_dict2, user_click_last_df, topk=200)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
